{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "22896b48",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter, Retry\n",
    "from decouple import AutoConfig\n",
    "import json\n",
    "import time\n",
    "import ast"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "2f4cfd7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = AutoConfig(\".env\")\n",
    "BEARER_TOKEN = config(\"BEARER_TOKEN\")\n",
    "HEADERS = {\"Authorization\": f\"Bearer {BEARER_TOKEN}\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "cd5ddac3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://stackoverflow.com/questions/312443/how-do-i-split-a-list-into-equally-sized-chunks\n",
    "def chunks(lst, n):\n",
    "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "16243e27",
   "metadata": {},
   "outputs": [],
   "source": [
    "USER_ENDPOINT = \"https://api.twitter.com/2/users\"\n",
    "\n",
    "def single_user_endpoint(user_id):\n",
    "    return f\"{USER_ENDPOINT}/{user_id}\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92062bc5",
   "metadata": {},
   "source": [
    "Get the list of author IDs from all tweets in our dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "bc60ff28",
   "metadata": {},
   "outputs": [],
   "source": [
    "authors = pd.read_csv(\"../shared_folder/Data/indo_dt_authors.csv\")\n",
    "author_list = authors.author_id.apply(str).to_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0135fcc9",
   "metadata": {},
   "source": [
    "___"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "92d74b36",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_data(author_id_list: list, user_endpoint: str, headers: dict):\n",
    "\n",
    "    user_data = []\n",
    "    errors = []\n",
    "    failed_request_users = []\n",
    "\n",
    "    with requests.Session() as s:\n",
    "\n",
    "        for i, c in enumerate(chunks(author_id_list, 100)):\n",
    "            params = {\"ids\": \",\".join(c)}\n",
    "            response = s.get(user_endpoint, \n",
    "                             params=params, \n",
    "                             headers=headers,)\n",
    "            \n",
    "            if response.status_code == 429:\n",
    "                time.sleep(120)\n",
    "                response = s.get(user_endpoint, \n",
    "                             params=params, \n",
    "                             headers=headers,)\n",
    "\n",
    "            if response.status_code != 200:\n",
    "                print(response.status_code)\n",
    "                failed_request_users.extend(c)\n",
    "            \n",
    "            json_response = response.json()\n",
    "            \n",
    "            if json_response.get(\"errors\"):\n",
    "                    errors.extend(json_response[\"errors\"])\n",
    "            \n",
    "            if json_response.get(\"data\"):\n",
    "                user_data.extend(json_response[\"data\"])\n",
    "            \n",
    "            if i % 10 == 0:\n",
    "                print(\"User data\", len(user_data), \n",
    "                      \"Failed request users\", len(failed_request_users), \n",
    "                      \"Errors\", len(errors),\n",
    "                      sep=\" \")\n",
    "            \n",
    "            time.sleep(3)\n",
    "            \n",
    "    return {\n",
    "        \"user_data\": user_data,\n",
    "        \"errors\": errors,\n",
    "        \"failed_request_users\": failed_request_users\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "ed2aca83",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User data 98 Failed request users 0 Errors 2\n",
      "User data 1058 Failed request users 0 Errors 42\n",
      "User data 2022 Failed request users 0 Errors 78\n",
      "User data 2983 Failed request users 0 Errors 117\n",
      "User data 3953 Failed request users 0 Errors 147\n",
      "User data 4921 Failed request users 0 Errors 179\n",
      "User data 5886 Failed request users 0 Errors 214\n",
      "User data 6861 Failed request users 0 Errors 239\n",
      "User data 7827 Failed request users 0 Errors 273\n",
      "User data 8791 Failed request users 0 Errors 309\n",
      "User data 9758 Failed request users 0 Errors 342\n",
      "User data 10720 Failed request users 0 Errors 380\n",
      "User data 11684 Failed request users 0 Errors 416\n",
      "User data 12651 Failed request users 0 Errors 449\n",
      "User data 13627 Failed request users 0 Errors 473\n",
      "User data 14600 Failed request users 0 Errors 500\n",
      "User data 15555 Failed request users 0 Errors 545\n",
      "User data 16515 Failed request users 0 Errors 585\n",
      "User data 17484 Failed request users 0 Errors 616\n",
      "User data 18454 Failed request users 0 Errors 646\n",
      "User data 19426 Failed request users 0 Errors 674\n",
      "User data 20391 Failed request users 0 Errors 709\n",
      "User data 21353 Failed request users 0 Errors 747\n",
      "User data 22324 Failed request users 0 Errors 776\n",
      "User data 23282 Failed request users 0 Errors 818\n",
      "User data 24243 Failed request users 0 Errors 857\n",
      "User data 25208 Failed request users 0 Errors 892\n",
      "User data 26178 Failed request users 0 Errors 922\n",
      "User data 27147 Failed request users 0 Errors 953\n",
      "User data 28114 Failed request users 0 Errors 986\n",
      "User data 29069 Failed request users 0 Errors 1031\n",
      "User data 30038 Failed request users 0 Errors 1062\n",
      "User data 30986 Failed request users 0 Errors 1114\n",
      "User data 31953 Failed request users 0 Errors 1147\n",
      "User data 32908 Failed request users 0 Errors 1192\n",
      "User data 33870 Failed request users 0 Errors 1230\n",
      "User data 34830 Failed request users 0 Errors 1270\n",
      "User data 35783 Failed request users 0 Errors 1317\n",
      "User data 36746 Failed request users 0 Errors 1354\n",
      "User data 37707 Failed request users 0 Errors 1393\n",
      "User data 38669 Failed request users 0 Errors 1431\n",
      "User data 39628 Failed request users 0 Errors 1472\n",
      "User data 40591 Failed request users 0 Errors 1509\n",
      "User data 41548 Failed request users 0 Errors 1552\n",
      "User data 42515 Failed request users 0 Errors 1585\n",
      "User data 43474 Failed request users 0 Errors 1626\n",
      "User data 44441 Failed request users 0 Errors 1659\n",
      "User data 45402 Failed request users 0 Errors 1698\n",
      "User data 46364 Failed request users 0 Errors 1736\n",
      "User data 47315 Failed request users 0 Errors 1785\n",
      "User data 48267 Failed request users 0 Errors 1833\n",
      "User data 49226 Failed request users 0 Errors 1874\n",
      "User data 50195 Failed request users 0 Errors 1905\n",
      "User data 51163 Failed request users 0 Errors 1937\n",
      "User data 53087 Failed request users 0 Errors 2013\n",
      "User data 54043 Failed request users 0 Errors 2057\n",
      "User data 55007 Failed request users 0 Errors 2093\n",
      "User data 55974 Failed request users 0 Errors 2126\n",
      "User data 56935 Failed request users 0 Errors 2165\n",
      "User data 57907 Failed request users 0 Errors 2193\n",
      "User data 58868 Failed request users 0 Errors 2232\n",
      "User data 59832 Failed request users 0 Errors 2268\n",
      "User data 60786 Failed request users 0 Errors 2314\n",
      "User data 61735 Failed request users 0 Errors 2365\n",
      "User data 62699 Failed request users 0 Errors 2401\n",
      "User data 63656 Failed request users 0 Errors 2444\n",
      "User data 64605 Failed request users 0 Errors 2495\n",
      "User data 65573 Failed request users 0 Errors 2527\n",
      "User data 66531 Failed request users 0 Errors 2569\n",
      "User data 67486 Failed request users 0 Errors 2614\n",
      "User data 68441 Failed request users 0 Errors 2659\n",
      "User data 69395 Failed request users 0 Errors 2705\n",
      "User data 70362 Failed request users 0 Errors 2738\n",
      "User data 71329 Failed request users 0 Errors 2771\n",
      "User data 72297 Failed request users 0 Errors 2803\n",
      "User data 73258 Failed request users 0 Errors 2842\n",
      "User data 74229 Failed request users 0 Errors 2871\n",
      "User data 75182 Failed request users 0 Errors 2918\n",
      "User data 76141 Failed request users 0 Errors 2959\n",
      "User data 77097 Failed request users 0 Errors 3003\n",
      "User data 78050 Failed request users 0 Errors 3050\n",
      "User data 78996 Failed request users 0 Errors 3104\n",
      "User data 79952 Failed request users 0 Errors 3148\n",
      "User data 80908 Failed request users 0 Errors 3192\n",
      "User data 81869 Failed request users 0 Errors 3231\n",
      "User data 82814 Failed request users 0 Errors 3286\n",
      "User data 83765 Failed request users 0 Errors 3335\n",
      "User data 84709 Failed request users 0 Errors 3391\n",
      "User data 85646 Failed request users 0 Errors 3454\n",
      "User data 86609 Failed request users 0 Errors 3491\n",
      "User data 87557 Failed request users 0 Errors 3543\n",
      "User data 88502 Failed request users 0 Errors 3598\n",
      "User data 89440 Failed request users 0 Errors 3660\n",
      "User data 90381 Failed request users 0 Errors 3719\n",
      "User data 91325 Failed request users 0 Errors 3775\n",
      "User data 92284 Failed request users 0 Errors 3816\n",
      "User data 93237 Failed request users 0 Errors 3863\n",
      "User data 94185 Failed request users 0 Errors 3915\n",
      "User data 95138 Failed request users 0 Errors 3962\n",
      "User data 96070 Failed request users 0 Errors 4030\n",
      "User data 97016 Failed request users 0 Errors 4084\n",
      "User data 97955 Failed request users 0 Errors 4145\n",
      "User data 98901 Failed request users 0 Errors 4199\n",
      "User data 99853 Failed request users 0 Errors 4247\n",
      "User data 100797 Failed request users 0 Errors 4303\n",
      "User data 101742 Failed request users 0 Errors 4358\n",
      "User data 102695 Failed request users 0 Errors 4405\n",
      "User data 103656 Failed request users 0 Errors 4444\n",
      "User data 104592 Failed request users 0 Errors 4508\n",
      "User data 105548 Failed request users 0 Errors 4552\n",
      "User data 106489 Failed request users 0 Errors 4611\n",
      "User data 107434 Failed request users 0 Errors 4666\n",
      "User data 108370 Failed request users 0 Errors 4730\n",
      "User data 109305 Failed request users 0 Errors 4795\n",
      "User data 110249 Failed request users 0 Errors 4851\n",
      "User data 111184 Failed request users 0 Errors 4916\n",
      "User data 112118 Failed request users 0 Errors 4982\n",
      "User data 113059 Failed request users 0 Errors 5041\n",
      "User data 113984 Failed request users 0 Errors 5116\n",
      "User data 114913 Failed request users 0 Errors 5187\n",
      "User data 115862 Failed request users 0 Errors 5238\n",
      "User data 116793 Failed request users 0 Errors 5307\n",
      "User data 117743 Failed request users 0 Errors 5357\n",
      "User data 118685 Failed request users 0 Errors 5415\n",
      "User data 119625 Failed request users 0 Errors 5475\n",
      "User data 120569 Failed request users 0 Errors 5531\n",
      "User data 121516 Failed request users 0 Errors 5584\n",
      "User data 122453 Failed request users 0 Errors 5647\n",
      "User data 123388 Failed request users 0 Errors 5712\n",
      "User data 124323 Failed request users 0 Errors 5777\n",
      "User data 125255 Failed request users 0 Errors 5845\n",
      "User data 126176 Failed request users 0 Errors 5924\n",
      "User data 127121 Failed request users 0 Errors 5979\n",
      "User data 128051 Failed request users 0 Errors 6049\n",
      "User data 128978 Failed request users 0 Errors 6122\n",
      "User data 129918 Failed request users 0 Errors 6182\n",
      "User data 130838 Failed request users 0 Errors 6262\n",
      "User data 131766 Failed request users 0 Errors 6334\n",
      "User data 132694 Failed request users 0 Errors 6406\n",
      "User data 133614 Failed request users 0 Errors 6486\n",
      "User data 134539 Failed request users 0 Errors 6561\n",
      "User data 135449 Failed request users 0 Errors 6651\n",
      "User data 136374 Failed request users 0 Errors 6726\n",
      "User data 137292 Failed request users 0 Errors 6808\n",
      "User data 138221 Failed request users 0 Errors 6879\n",
      "User data 139149 Failed request users 0 Errors 6951\n",
      "User data 140066 Failed request users 0 Errors 7034\n",
      "User data 140966 Failed request users 0 Errors 7134\n",
      "User data 141891 Failed request users 0 Errors 7209\n",
      "User data 142767 Failed request users 0 Errors 7333\n",
      "User data 143653 Failed request users 0 Errors 7447\n",
      "User data 144540 Failed request users 0 Errors 7560\n",
      "User data 145448 Failed request users 0 Errors 7652\n",
      "User data 146353 Failed request users 0 Errors 7747\n",
      "User data 147257 Failed request users 0 Errors 7843\n",
      "User data 148147 Failed request users 0 Errors 7953\n",
      "User data 149037 Failed request users 0 Errors 8063\n",
      "User data 149930 Failed request users 0 Errors 8170\n",
      "User data 150564 Failed request users 0 Errors 8536\n",
      "User data 151314 Failed request users 0 Errors 8786\n",
      "User data 152166 Failed request users 0 Errors 8934\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User data 153003 Failed request users 0 Errors 9097\n",
      "User data 153812 Failed request users 0 Errors 9288\n",
      "User data 154489 Failed request users 0 Errors 9611\n",
      "User data 155081 Failed request users 0 Errors 10019\n",
      "User data 155636 Failed request users 0 Errors 10464\n",
      "User data 156268 Failed request users 0 Errors 10832\n",
      "User data 156869 Failed request users 0 Errors 11231\n",
      "User data 157567 Failed request users 0 Errors 11533\n",
      "User data 158535 Failed request users 0 Errors 11565\n",
      "User data 159517 Failed request users 0 Errors 11583\n",
      "User data 160493 Failed request users 0 Errors 11607\n",
      "User data 161473 Failed request users 0 Errors 11627\n",
      "User data 162445 Failed request users 0 Errors 11655\n",
      "User data 163412 Failed request users 0 Errors 11688\n",
      "User data 164387 Failed request users 0 Errors 11713\n",
      "User data 165360 Failed request users 0 Errors 11740\n",
      "User data 166335 Failed request users 0 Errors 11765\n",
      "User data 167312 Failed request users 0 Errors 11788\n",
      "User data 168281 Failed request users 0 Errors 11819\n",
      "User data 169242 Failed request users 0 Errors 11858\n",
      "User data 170219 Failed request users 0 Errors 11881\n",
      "User data 171194 Failed request users 0 Errors 11906\n",
      "User data 172159 Failed request users 0 Errors 11941\n",
      "User data 173133 Failed request users 0 Errors 11967\n",
      "User data 174119 Failed request users 0 Errors 11981\n",
      "User data 175093 Failed request users 0 Errors 12007\n",
      "User data 176057 Failed request users 0 Errors 12043\n",
      "User data 177039 Failed request users 0 Errors 12061\n",
      "User data 178020 Failed request users 0 Errors 12080\n",
      "User data 178989 Failed request users 0 Errors 12111\n",
      "User data 179962 Failed request users 0 Errors 12138\n",
      "User data 180934 Failed request users 0 Errors 12166\n",
      "User data 181910 Failed request users 0 Errors 12190\n",
      "User data 182876 Failed request users 0 Errors 12224\n",
      "User data 183848 Failed request users 0 Errors 12252\n",
      "User data 184829 Failed request users 0 Errors 12271\n",
      "User data 185807 Failed request users 0 Errors 12293\n",
      "User data 186782 Failed request users 0 Errors 12318\n",
      "User data 187753 Failed request users 0 Errors 12347\n",
      "User data 188719 Failed request users 0 Errors 12381\n",
      "User data 189685 Failed request users 0 Errors 12415\n",
      "User data 190666 Failed request users 0 Errors 12434\n",
      "User data 191641 Failed request users 0 Errors 12459\n",
      "User data 192614 Failed request users 0 Errors 12486\n",
      "User data 193569 Failed request users 0 Errors 12531\n",
      "User data 194547 Failed request users 0 Errors 12553\n",
      "User data 195530 Failed request users 0 Errors 12570\n",
      "User data 196498 Failed request users 0 Errors 12602\n",
      "User data 197469 Failed request users 0 Errors 12631\n",
      "User data 198440 Failed request users 0 Errors 12660\n",
      "User data 199411 Failed request users 0 Errors 12689\n",
      "User data 200344 Failed request users 0 Errors 12756\n",
      "User data 201284 Failed request users 0 Errors 12816\n",
      "User data 202251 Failed request users 0 Errors 12849\n",
      "User data 203222 Failed request users 0 Errors 12878\n",
      "User data 204193 Failed request users 0 Errors 12907\n",
      "User data 205167 Failed request users 0 Errors 12933\n",
      "User data 206145 Failed request users 0 Errors 12955\n",
      "User data 207118 Failed request users 0 Errors 12982\n",
      "User data 208092 Failed request users 0 Errors 13008\n",
      "User data 209065 Failed request users 0 Errors 13035\n",
      "User data 210041 Failed request users 0 Errors 13059\n",
      "User data 211014 Failed request users 0 Errors 13086\n",
      "User data 211989 Failed request users 0 Errors 13111\n",
      "User data 212964 Failed request users 0 Errors 13136\n",
      "User data 213938 Failed request users 0 Errors 13162\n",
      "User data 214901 Failed request users 0 Errors 13199\n",
      "User data 215876 Failed request users 0 Errors 13224\n",
      "User data 216853 Failed request users 0 Errors 13247\n",
      "User data 217821 Failed request users 0 Errors 13279\n",
      "User data 218791 Failed request users 0 Errors 13309\n",
      "User data 219762 Failed request users 0 Errors 13338\n",
      "User data 220731 Failed request users 0 Errors 13369\n",
      "User data 221709 Failed request users 0 Errors 13391\n",
      "User data 222685 Failed request users 0 Errors 13415\n",
      "User data 223651 Failed request users 0 Errors 13449\n",
      "User data 224615 Failed request users 0 Errors 13485\n",
      "User data 225590 Failed request users 0 Errors 13510\n",
      "User data 226559 Failed request users 0 Errors 13541\n",
      "User data 227526 Failed request users 0 Errors 13574\n",
      "User data 228493 Failed request users 0 Errors 13607\n",
      "User data 229455 Failed request users 0 Errors 13645\n",
      "User data 230425 Failed request users 0 Errors 13675\n",
      "User data 231406 Failed request users 0 Errors 13694\n",
      "User data 232381 Failed request users 0 Errors 13719\n",
      "User data 233360 Failed request users 0 Errors 13740\n",
      "User data 234333 Failed request users 0 Errors 13767\n",
      "User data 235304 Failed request users 0 Errors 13796\n",
      "User data 236276 Failed request users 0 Errors 13824\n",
      "User data 237254 Failed request users 0 Errors 13846\n",
      "User data 238225 Failed request users 0 Errors 13875\n",
      "User data 239189 Failed request users 0 Errors 13911\n",
      "User data 240156 Failed request users 0 Errors 13944\n",
      "User data 241138 Failed request users 0 Errors 13962\n",
      "User data 242116 Failed request users 0 Errors 13984\n",
      "User data 243098 Failed request users 0 Errors 14002\n",
      "User data 244066 Failed request users 0 Errors 14034\n",
      "User data 245042 Failed request users 0 Errors 14058\n",
      "User data 246022 Failed request users 0 Errors 14078\n",
      "User data 246987 Failed request users 0 Errors 14113\n",
      "User data 247973 Failed request users 0 Errors 14127\n",
      "User data 248940 Failed request users 0 Errors 14160\n",
      "User data 249912 Failed request users 0 Errors 14188\n",
      "User data 250893 Failed request users 0 Errors 14207\n",
      "User data 251860 Failed request users 0 Errors 14240\n",
      "User data 252828 Failed request users 0 Errors 14272\n",
      "User data 253798 Failed request users 0 Errors 14302\n",
      "User data 254775 Failed request users 0 Errors 14325\n",
      "User data 255740 Failed request users 0 Errors 14360\n",
      "User data 256712 Failed request users 0 Errors 14388\n",
      "User data 257681 Failed request users 0 Errors 14419\n",
      "User data 258655 Failed request users 0 Errors 14445\n",
      "User data 259636 Failed request users 0 Errors 14464\n",
      "User data 260618 Failed request users 0 Errors 14482\n",
      "User data 261592 Failed request users 0 Errors 14508\n",
      "User data 262571 Failed request users 0 Errors 14529\n",
      "User data 263546 Failed request users 0 Errors 14554\n",
      "User data 264524 Failed request users 0 Errors 14576\n",
      "User data 265506 Failed request users 0 Errors 14594\n",
      "User data 266484 Failed request users 0 Errors 14616\n",
      "User data 267444 Failed request users 0 Errors 14656\n",
      "User data 268419 Failed request users 0 Errors 14681\n",
      "User data 269391 Failed request users 0 Errors 14709\n",
      "User data 270367 Failed request users 0 Errors 14733\n",
      "User data 271337 Failed request users 0 Errors 14763\n",
      "User data 272318 Failed request users 0 Errors 14782\n",
      "User data 273287 Failed request users 0 Errors 14813\n",
      "User data 274260 Failed request users 0 Errors 14840\n",
      "User data 275231 Failed request users 0 Errors 14869\n",
      "User data 276203 Failed request users 0 Errors 14897\n",
      "User data 277181 Failed request users 0 Errors 14919\n",
      "User data 278161 Failed request users 0 Errors 14939\n",
      "User data 279130 Failed request users 0 Errors 14970\n",
      "User data 280096 Failed request users 0 Errors 15004\n",
      "User data 281067 Failed request users 0 Errors 15033\n",
      "User data 282031 Failed request users 0 Errors 15069\n",
      "User data 283003 Failed request users 0 Errors 15097\n",
      "User data 283975 Failed request users 0 Errors 15125\n",
      "User data 284937 Failed request users 0 Errors 15163\n",
      "User data 285907 Failed request users 0 Errors 15193\n",
      "User data 286865 Failed request users 0 Errors 15235\n",
      "User data 287841 Failed request users 0 Errors 15259\n",
      "User data 288807 Failed request users 0 Errors 15293\n",
      "User data 289775 Failed request users 0 Errors 15325\n",
      "User data 290745 Failed request users 0 Errors 15355\n",
      "User data 291713 Failed request users 0 Errors 15387\n",
      "User data 292677 Failed request users 0 Errors 15423\n",
      "User data 293646 Failed request users 0 Errors 15454\n",
      "User data 294613 Failed request users 0 Errors 15487\n",
      "User data 295583 Failed request users 0 Errors 15517\n",
      "User data 296556 Failed request users 0 Errors 15544\n",
      "User data 297514 Failed request users 0 Errors 15586\n",
      "User data 298482 Failed request users 0 Errors 15618\n",
      "User data 299444 Failed request users 0 Errors 15656\n",
      "User data 300412 Failed request users 0 Errors 15688\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User data 301373 Failed request users 0 Errors 15727\n",
      "User data 302332 Failed request users 0 Errors 15768\n",
      "User data 303296 Failed request users 0 Errors 15804\n",
      "User data 304259 Failed request users 0 Errors 15841\n",
      "User data 305219 Failed request users 0 Errors 15881\n",
      "User data 306189 Failed request users 0 Errors 15911\n",
      "User data 307150 Failed request users 0 Errors 15950\n",
      "User data 308124 Failed request users 0 Errors 15976\n",
      "User data 309092 Failed request users 0 Errors 16008\n",
      "User data 310051 Failed request users 0 Errors 16049\n",
      "User data 311006 Failed request users 0 Errors 16094\n",
      "User data 311966 Failed request users 0 Errors 16134\n",
      "User data 312930 Failed request users 0 Errors 16170\n",
      "User data 313894 Failed request users 0 Errors 16206\n",
      "User data 314858 Failed request users 0 Errors 16242\n",
      "User data 315826 Failed request users 0 Errors 16274\n",
      "User data 316781 Failed request users 0 Errors 16319\n",
      "User data 317738 Failed request users 0 Errors 16362\n",
      "User data 318695 Failed request users 0 Errors 16405\n",
      "User data 319654 Failed request users 0 Errors 16446\n",
      "User data 320619 Failed request users 0 Errors 16481\n",
      "User data 321577 Failed request users 0 Errors 16523\n",
      "User data 322540 Failed request users 0 Errors 16560\n",
      "User data 323497 Failed request users 0 Errors 16603\n",
      "User data 324462 Failed request users 0 Errors 16638\n",
      "User data 325414 Failed request users 0 Errors 16686\n",
      "User data 326371 Failed request users 0 Errors 16729\n",
      "User data 327326 Failed request users 0 Errors 16774\n",
      "User data 328290 Failed request users 0 Errors 16810\n",
      "User data 329256 Failed request users 0 Errors 16844\n",
      "User data 330223 Failed request users 0 Errors 16877\n",
      "User data 331192 Failed request users 0 Errors 16908\n",
      "User data 332163 Failed request users 0 Errors 16937\n",
      "User data 333131 Failed request users 0 Errors 16969\n",
      "User data 334097 Failed request users 0 Errors 17003\n",
      "User data 335066 Failed request users 0 Errors 17034\n",
      "User data 336024 Failed request users 0 Errors 17076\n",
      "User data 336991 Failed request users 0 Errors 17109\n",
      "User data 337958 Failed request users 0 Errors 17142\n",
      "User data 338915 Failed request users 0 Errors 17185\n",
      "User data 339887 Failed request users 0 Errors 17213\n",
      "User data 340859 Failed request users 0 Errors 17241\n",
      "User data 341823 Failed request users 0 Errors 17277\n",
      "User data 342780 Failed request users 0 Errors 17320\n"
     ]
    }
   ],
   "source": [
    "responses = get_user_data(author_list, user_endpoint=USER_ENDPOINT, headers=HEADERS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "2bb3a89b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "360897"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(author_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "82caf235",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "360897"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(responses[\"user_data\"]) + len(responses[\"errors\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "id": "3c3f2a5a",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"2022-07-28_user_status.json\", \"w\") as f:\n",
    "    json.dump(responses, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "id": "092449be",
   "metadata": {},
   "outputs": [],
   "source": [
    "users_df = pd.DataFrame(responses[\"user_data\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "47367bbf",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_df = pd.DataFrame(responses[\"errors\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "8bc7a419",
   "metadata": {},
   "outputs": [],
   "source": [
    "errors_df[[\"value\", \"detail\", \"title\"]].to_csv(\"tweets/2022-07-28_not_found_or_suspended_users.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4a7c0654",
   "metadata": {},
   "source": [
    "___"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
